#coding=utf-8
#encoding: utf-8
'''
Created on 2015年4月16日

@author: leoZ
'''

from datetime import date
import logging
import os

from com.cyou.receive import FtpClient
from com.cyou.repository.EtlFtpSource import EtlFtpSourceDao
from com.cyou.repository.EtlJob import EtlJobDao
from com.cyou.repository.EtlJobSource import EtlJobSourceDao

logging.basicConfig()
log = logging.getLogger("FtpReceive")
log.setLevel(logging.DEBUG)

class FtpReceive(object):
    #初始化
    def init(self, hqlReceive):
        self.sourceDict = {}
        self.curTime = hqlReceive.curTime
#        self.ftpClient = FtpClient.FtpClient()
#        self.ftpClient.initFtpServer(0,"10.1.9.28",21,"etl","etl")
    #获取需要检验的文件列表
    def getReceiverSourceList(self):
        log.debug("开始获取监控日志列表")
        self.sourceDict.clear()
        job = EtlJobDao()
        jobSource = EtlJobSourceDao()
        ftpSource = EtlFtpSourceDao()
        #从DB中获取监控数据
        jobs = job.loadAll()
        for job in jobs:
            log.debug("获取需要检验的文件:%s-%s %s" % (job.etlSystemName, job.etlJobName, job.lastJobStatus))
            if job.lastJobStatus != "Ready":
                continue
            etlSystemName = job.etlSystemName
            etlJobName = job.etlJobName
            fullJobName = etlSystemName + etlJobName
            jobSources = jobSource.loadAllJobSourceByJobName(etlSystemName,etlJobName)
            for source in jobSources:
                ftps = ftpSource.loadAllFtpJobSourceByJobSource(source)
                for ftp in ftps:
                    self.sourceDict[fullJobName] = ftp      
        log.debug("获取监控日志列表完成")
    #替换路径中的日期
    def repalceMonitorPaht(self,monitorPath):
        return monitorPath.replace("YYYY-MM-DD", str(self.curTime))
    #创建一个FTP连接
    def createFtpClient(self, ftpEntity):
        self.ftpClient = FtpClient.FtpClient()
        self.ftpClient.initFtpServer(0, ftpEntity.ftpIp,ftpEntity.ftpPort,ftpEntity.ftpUser,ftpEntity.ftpPassword)
        return self.ftpClient
    #查看文件是否是完整的。(非.tmp结尾的为完整的文件）
    def isFinishFile(self,ftpEntity):
        ftpClient = self.createFtpClient(ftpEntity)
        remoteFilePath = ftpEntity.ftpFilePath[0:ftpEntity.ftpFilePath.rindex("/")]
        remoteFileName = ftpEntity.ftpFilePath[ftpEntity.ftpFilePath.rindex("/") + 1:]
        log.debug("检验是否可以下载%s目录下的文件%s" % (remoteFilePath, remoteFileName))
        ftpClient.ftp.cwd(remoteFilePath)
        try:
            size = ftpClient.ftp.size(remoteFileName)
        except Exception:
            log.debug("文件还没有产生完成。下次再验证")
            ftpClient.ftp.close()
            return False
        log.debug("文件%s大小为:%d" % (ftpEntity.ftpFilePath, size))
        ftpClient.ftp.close()
        if size > 0:
            return True
    #检验文件是否可以下载
    def checkFileValid(self, ftpEntity):
        #1.替换路径中的日期
        ftpFilePath = ftpEntity.ftpFilePath
        ftpEntity.ftpFilePath = self.repalceMonitorPaht(ftpFilePath)
        log.debug("检验是否可以下载文件：" + ftpFilePath)
        #2.TODO 原想法（根据实际情况进行判断,如果生成新的一天的日志，就说明前一天的是完整的。）
        #但是由于日志每天只同步一次，所以改为未完成时加后缀来实现 syslog.log.tmp -> syslog.log
        return self.isFinishFile(ftpEntity)
    #设置接受数据的目录结构。之后从醒置文件中读取
    def getDstPath(self,ftpEntity):
        jobsourceEntity = self.getJobSourceBy(ftpEntity)
        return "E:/data/receive/%s/%s" % (self.curTime, jobsourceEntity.source)
    #获取JobSource，以便找到下载的位置
    def getJobSourceBy(self,ftpEntity):
        etlSystemName = ftpEntity.etlSystemName
        etlJobName = ftpEntity.etlJobName
        jobSource = EtlJobSourceDao()
        jobSources = jobSource.loadAllJobSourceByJobName(etlSystemName, etlJobName)
        for jobsourceEntity in jobSources:
            return jobsourceEntity
        return None
    #接收文件
    def receiveData(self, ftpEntity):
        monitorPath = ftpEntity.ftpFilePath
        log.debug("接收数据" + monitorPath + "开始")
        fileName = monitorPath[monitorPath.rindex("/")+1:]
        srcPath = monitorPath[0:monitorPath.rindex("/")]
        dstPath = self.getDstPath(ftpEntity)
        ftpClient = self.createFtpClient(ftpEntity)
        ftpClient.downloadFile(fileName, srcPath, dstPath)
        #更新单个文件的状态标记
        log.debug("接收数据" + fileName + "完成")
    #检查同一模块下相关数据时候全部就绪
    def checkRelationDataAllReady(self, ftpEntity):
        log.debug("开始检查文件[" + ftpEntity.etlJobName +"]相关的文件是否全部下载完成了")
        etlSystemName = ftpEntity.etlSystemName
        etlJobName = ftpEntity.etlJobName
        etlJobSourceDao = EtlJobSourceDao()
        etlFtpSourceDao = EtlFtpSourceDao()
        #首先通过ftpentity来获得jobsource中的source。然后通过source来获取同一个source下的所有ftpEntity。
        #因为ftpEntity中有文件路径。然后通过文件路径，获得文件的名字。
        #最后逐个检查同一个Source下的文件是否都已经存在。如果存在，则返回成功。如果有不存在的反回失败
        jobSources = etlJobSourceDao.loadAllJobSourceByJobName(etlSystemName, etlJobName)
        if len(jobSources) == 1:
            source = jobSources[0].source
            allSource = etlJobSourceDao.loadAllJobSourceBySource(source)
            for oneSource in allSource:
                ftpJobs = etlFtpSourceDao.loadAllJobSourceByJobName(oneSource.etlSystemName, oneSource.etlJobName)
                if len(ftpJobs) == 1:
                    path = self.getDstPath(ftpJobs[0])
                    tmp = ftpJobs[0].ftpFilePath
                    tmp = self.repalceMonitorPaht(tmp)
                    fileName = tmp[tmp.rindex("/"):]
                    if not os.path.exists(path + fileName):
                        log.debug("不存在文件：" + path + fileName)
                        return False
                    log.debug("已经存在文件：" + path + fileName)
        #需要通过数据库，获取依赖文件列表，并检查依赖数据是否完成
        log.debug("完成检查文件[" + ftpEntity.etlJobName +"]相关的文件是否全部下载完成了")
        return True
    #内部帮助方法。ftpEntity -> jobsource
    def getJobSourceByFtpEntity(self,ftpEntity): 
        etlSystemName = ftpEntity.etlSystemName
        etlJobName = ftpEntity.etlJobName
        etlJobSourceDao = EtlJobSourceDao()
        jobSources = etlJobSourceDao.loadAllJobSourceByJobName(etlSystemName, etlJobName)
        if len(jobSources) == 1:
            return jobSources[0]
        return None
    #用于生成就绪文件
    #就绪文件产生的原理是通过ftpEntity，获取对应的source。然后产生source对应的文件。生成位置在receive目录下
    def createReadyFile(self, ftpEntity):
        log.debug("开始生成就绪文件")
        #获取jobSource中的source
        jobSource =  self.getJobSourceByFtpEntity(ftpEntity)
        #产生生成就绪文件的地址
        if jobSource is not None:
            readyFile =  self.getDstPath(ftpEntity) + "/" + jobSource.source + ".ready"
            log.debug("生成就绪文件为:" + readyFile)
            #写入就绪文件
            f=open(readyFile,'w') 
            f.close()
            log.debug("完成生成就绪文件")
            return True
        log.error("出错。没有找到jobSource.")
        return False
    
    #将任务况态设置为完成。
    def updateJobFinish(self, ftpEntity):
        #更新job的执行况态
        job = EtlJobDao()
        job.updateJobFinish(ftpEntity.etlSystemName, ftpEntity.etlJobName)
        log.debug("Job:%s-%s更新执行况态为done" % (ftpEntity.etlSystemName, ftpEntity.etlJobName))
    #处理数据收集任务
    def process(self):
        log.debug("FTP接收本次执行开始")
        self.getReceiverSourceList()
        for key in self.sourceDict:
            ftpEntity = self.sourceDict[key]
            if self.checkFileValid(ftpEntity):
                log.debug("开始下载")
                self.receiveData(ftpEntity)
                #下载完成后，将任务况态设置为完成。
                self.updateJobFinish(ftpEntity)
                if self.checkRelationDataAllReady(ftpEntity):
                    log.debug("生成就绪文件")
                    self.createReadyFile(ftpEntity)
        log.debug("FTP接收本次执行完毕")

if __name__ == '__main__':
    ftpReceive = FtpReceive()
    ftpReceive.curTime = date.today()
    dao = EtlFtpSourceDao()
    ftpEntity = dao.loadAll()[0]
    #print ftpReceive.checkRelationDataAllReady(ftpEntity)
    ftpReceive.createReadyFile(ftpEntity)